From abd30192c04df508da0c4c3d7803c95125a59e51 Mon Sep 17 00:00:00 2001 From: "emellor@leeni.uk.xensource.com" Date: Fri, 14 Apr 2006 21:22:09 +0100 Subject: [PATCH] This patch enables external devices, such as for example a mounted hard drive image or a TPM, to be migrated to a remote machine. The patch hooks into the checkpointing (XendCheckpoint.py) code and performs migration in 4 different steps: In a 1st step (step = 0 in the code) migration of all devices of a domain is 'tested', that means their driver implementations (blkif.py, netif.py, tpmif.py, usbif.py, pciif.py) are queried whether migration is possible at all. Currently all device representations respond with a 'yes' (=0), although probably a VM mounting a hard drive partition should respond with a 'no' (-1) already. This first step is a quick check to see whether devices can be migrated. The 2nd step is to do whatever can be done before the domain is suspended. At this point migration of the device could be initiated, if at all possible. The 3rd step is to migrate a device after the domain has been suspended, meaning that it is not scheduled anymore and the VM is 'settled'. All devices are called again and a good implementation would initiate the migration in a background process to achieve as much concurrency as possible. The 4th step is to synchronize with the 3rd step. At this point the implementor has to make sure that anything that was initiated in step 3 has completed. Once all steps 4 have been processed, the VM will resume on the remove machine. I have implemented hooks for migration of a virtual TPM in xen/xend/server/tpmif.py. These hooks call a configurable external migration tool using the os.popen() call with a fixed command line parameter set. The implementation refuses to migrate a VM attached to a virtual TPM if no tool has been provided for migration. All other devices do not currently overload the 'migrate' method defined in the DevController.py and therefore will just let migration happen. I have added hooks for error recovery such that whatever part of migration has been initiated can be rolled back when any of the devices fail to migrate in one of the steps. The interface (in tpmif.py) to the external application now uses os.popen() to allow error handling by reading the application's output. Signed-off-by: Stefan Berger --- tools/examples/xend-config.sxp | 3 ++ tools/python/xen/xend/XendCheckpoint.py | 6 ++- tools/python/xen/xend/XendDomain.py | 9 ++-- tools/python/xen/xend/XendDomainInfo.py | 32 +++++++++++++ tools/python/xen/xend/XendRoot.py | 6 +++ tools/python/xen/xend/server/DevController.py | 35 ++++++++++++++ tools/python/xen/xend/server/tpmif.py | 48 +++++++++++++++++++ 7 files changed, 135 insertions(+), 4 deletions(-) diff --git a/tools/examples/xend-config.sxp b/tools/examples/xend-config.sxp index 7dd74f793a..00ae08ca0d 100644 --- a/tools/examples/xend-config.sxp +++ b/tools/examples/xend-config.sxp @@ -127,3 +127,6 @@ # Whether to enable core-dumps when domains crash. #(enable-dump no) + +# The tool used for initiating virtual TPM migration +#(external-migration-tool '') diff --git a/tools/python/xen/xend/XendCheckpoint.py b/tools/python/xen/xend/XendCheckpoint.py index 63d5821a00..f9073a4823 100644 --- a/tools/python/xen/xend/XendCheckpoint.py +++ b/tools/python/xen/xend/XendCheckpoint.py @@ -53,7 +53,7 @@ def read_exact(fd, size, errmsg): -def save(fd, dominfo, live): +def save(fd, dominfo, live, dst): write_exact(fd, SIGNATURE, "could not write guest state file: signature") config = sxp.to_string(dominfo.sxpr()) @@ -65,6 +65,8 @@ def save(fd, dominfo, live): dominfo.setName('migrating-' + domain_name) try: + dominfo.migrateDevices(live, dst, 1, domain_name) + write_exact(fd, pack("!i", len(config)), "could not write guest state file: config len") write_exact(fd, config, "could not write guest state file: config") @@ -85,7 +87,9 @@ def save(fd, dominfo, live): log.debug("Suspending %d ...", dominfo.getDomid()) dominfo.shutdown('suspend') dominfo.waitForShutdown() + dominfo.migrateDevices(live, dst, 2, domain_name) log.info("Domain %d suspended.", dominfo.getDomid()) + dominfo.migrateDevices(live, dst, 3, domain_name) tochild.write("done\n") tochild.flush() log.debug('Written done') diff --git a/tools/python/xen/xend/XendDomain.py b/tools/python/xen/xend/XendDomain.py index a762f88c6a..9af3bb75a8 100644 --- a/tools/python/xen/xend/XendDomain.py +++ b/tools/python/xen/xend/XendDomain.py @@ -405,6 +405,9 @@ class XendDomain: if dominfo.getDomid() == PRIV_DOMAIN: raise XendError("Cannot migrate privileged domain %i" % domid) + """ The following call may raise a XendError exception """ + dominfo.testMigrateDevices(live, dst) + if port == 0: port = xroot.get_xend_relocation_port() try: @@ -414,8 +417,8 @@ class XendDomain: raise XendError("can't connect: %s" % err[1]) sock.send("receive\n") - sock.recv(80) - XendCheckpoint.save(sock.fileno(), dominfo, live) + sock.recv(80) + XendCheckpoint.save(sock.fileno(), dominfo, live, dst) def domain_save(self, domid, dst): @@ -435,7 +438,7 @@ class XendDomain: fd = os.open(dst, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) try: # For now we don't support 'live checkpoint' - return XendCheckpoint.save(fd, dominfo, False) + return XendCheckpoint.save(fd, dominfo, False, dst) finally: os.close(fd) except OSError, ex: diff --git a/tools/python/xen/xend/XendDomainInfo.py b/tools/python/xen/xend/XendDomainInfo.py index 1fc9137e40..c7c26c1d9c 100644 --- a/tools/python/xen/xend/XendDomainInfo.py +++ b/tools/python/xen/xend/XendDomainInfo.py @@ -1395,6 +1395,38 @@ class XendDomainInfo: if self.image: self.image.createDeviceModel() + ## public: + + def testMigrateDevices(self, live, dst): + """ Notify all device about intention of migration + @raise: XendError for a device that cannot be migrated + """ + for (n, c) in self.info['device']: + rc = self.migrateDevice(n, c, live, dst, 0) + if rc != 0: + raise XendError("Device of type '%s' refuses migration." % n) + + def migrateDevices(self, live, dst, step, domName=''): + """Notify the devices about migration + """ + ctr = 0 + try: + for (n, c) in self.info['device']: + self.migrateDevice(n, c, live, dst, step, domName) + ctr = ctr + 1 + except: + for (n, c) in self.info['device']: + if ctr == 0: + step = step - 1 + ctr = ctr - 1 + self.recoverMigrateDevice(n, c, live, dst, step, domName) + raise + + def migrateDevice(self, deviceClass, deviceConfig, live, dst, step, domName=''): + return self.getDeviceController(deviceClass).migrate(deviceConfig, live, dst, step, domName) + + def recoverMigrateDevice(self, deviceClass, deviceConfig, live, dst, step, domName=''): + return self.getDeviceController(deviceClass).recover_migrate(deviceConfig, live, dst, step, domName) def waitForDevices(self): """Wait for this domain's configured devices to connect. diff --git a/tools/python/xen/xend/XendRoot.py b/tools/python/xen/xend/XendRoot.py index dc3e64f38c..5f0026907e 100644 --- a/tools/python/xen/xend/XendRoot.py +++ b/tools/python/xen/xend/XendRoot.py @@ -86,6 +86,9 @@ class XendRoot: server (deprecated).""" xend_unix_server_default = 'no' + """Default external migration tool """ + external_migration_tool_default = '' + """Default path the unix-domain server listens at.""" xend_unix_path_default = '/var/lib/xend/xend-socket' @@ -250,6 +253,9 @@ class XendRoot: else: return None + def get_external_migration_tool(self): + """@return the name of the tool to handle virtual TPM migration.""" + return self.get_config_value('external-migration-tool', self.external_migration_tool_default) def get_enable_dump(self): return self.get_config_bool('enable-dump', 'no') diff --git a/tools/python/xen/xend/server/DevController.py b/tools/python/xen/xend/server/DevController.py index 71c54596f5..a0de4ae793 100644 --- a/tools/python/xen/xend/server/DevController.py +++ b/tools/python/xen/xend/server/DevController.py @@ -267,6 +267,41 @@ class DevController: raise NotImplementedError() + def migrate(self, deviceConfig, live, dst, step, domName): + """ Migration of a device. The 'live' parameter indicates + whether the device is live-migrated (live=1). 'dst' then gives + the hostname of the machine to migrate to. + This function is called for 4 steps: + If step == 0: Check whether the device is ready to be migrated + or can at all be migrated; return a '-1' if + the device is NOT ready, a '0' otherwise. If it is + not ready ( = not possible to migrate this device), + migration will not take place. + step == 1: Called immediately after step 0; migration + of the kernel has started; + step == 2: Called after the suspend has been issued + to the domain and the domain is not scheduled anymore. + Synchronize with what was started in step 1, if necessary. + Now the device should initiate its transfer to the + given target. Since there might be more than just + one device initiating a migration, this step should + put the process performing the transfer into the + background and return immediately to achieve as much + concurrency as possible. + step == 3: Synchronize with the migration of the device that + was initiated in step 2. + Make sure that the migration has finished and only + then return from the call. + """ + return 0 + + + def recover_migrate(self, deviceConfig, list, dst, step, domName): + """ Recover from device migration. The given step was the + last one that was successfully executed. + """ + return 0 + def getDomid(self): """Stub to {@link XendDomainInfo.getDomid}, for use by our diff --git a/tools/python/xen/xend/server/tpmif.py b/tools/python/xen/xend/server/tpmif.py index aa810390df..338bcbffc2 100644 --- a/tools/python/xen/xend/server/tpmif.py +++ b/tools/python/xen/xend/server/tpmif.py @@ -23,9 +23,17 @@ from xen.xend import sxp from xen.xend.XendLogging import log +from xen.xend.XendError import XendError +from xen.xend import XendRoot from xen.xend.server.DevController import DevController +import os +import re + + +xroot = XendRoot.instance() + class TPMifController(DevController): """TPM interface controller. Handles all TPM devices for a domain. @@ -61,3 +69,43 @@ class TPMifController(DevController): result.append(['instance', instance]) return result + + def migrate(self, deviceConfig, live, dst, step, domName): + """@see DevContoller.migrate""" + if live: + tool = xroot.get_external_migration_tool() + if tool != '': + log.info("Request to live-migrate device to %s. step=%d.", + dst, step) + + if step == 0: + """Assuming for now that everything is ok and migration + with the given tool can proceed. + """ + return 0 + else: + fd = os.popen("%s -type vtpm -step %d -host %s -domname %s" % + (tool, step, dst, domName), + 'r') + for line in fd.readlines(): + mo = re.search('Error', line) + if mo: + raise XendError("vtpm: Fatal error in migration step %d." % + step) + return 0 + else: + log.debug("External migration tool not in configuration.") + return -1 + return 0 + + def recover_migrate(self, deviceConfig, live, dst, step, domName): + """@see DevContoller.recover_migrate""" + if live: + tool = xroot.get_external_migration_tool() + if tool != '': + log.info("Request to recover live-migrated device. last good step=%d.", + step) + fd = os.popen("%s -type vtpm -step %d -host %s -domname %s -recover" % + (tool, step, dst, domName), + 'r') + return 0 -- 2.30.2